<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><title>虚拟机+Ubuntu，搭建hadoop集群 | 细粒丁の小窝</title><meta name="keywords" content="虚拟机,Linux,Hadoop"><meta name="author" content="细粒丁"><meta name="copyright" content="细粒丁"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="虚拟机：VirtualBox-6.1.18Ubuntu：16.04桌面版">
<meta property="og:type" content="article">
<meta property="og:title" content="虚拟机+Ubuntu，搭建hadoop集群">
<meta property="og:url" content="https://tenyding.cn/%E7%B2%BE%E9%80%9A%E5%90%84%E8%BD%AF%E4%BB%B6%E7%9A%84%E5%AE%89%E8%A3%85/%E8%99%9A%E6%8B%9F%E6%9C%BA+Ubuntu%EF%BC%8C%E6%90%AD%E5%BB%BAhadoop%E9%9B%86%E7%BE%A4/index.html">
<meta property="og:site_name" content="细粒丁の小窝">
<meta property="og:description" content="虚拟机：VirtualBox-6.1.18Ubuntu：16.04桌面版">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://tenyding.cn/image/bg9.jpg">
<meta property="article:published_time" content="2021-03-13T21:29:56.000Z">
<meta property="article:modified_time" content="2022-03-22T20:26:12.339Z">
<meta property="article:author" content="细粒丁">
<meta property="article:tag" content="虚拟机">
<meta property="article:tag" content="Linux">
<meta property="article:tag" content="Hadoop">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://tenyding.cn/image/bg9.jpg"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="https://tenyding.cn/%E7%B2%BE%E9%80%9A%E5%90%84%E8%BD%AF%E4%BB%B6%E7%9A%84%E5%AE%89%E8%A3%85/%E8%99%9A%E6%8B%9F%E6%9C%BA+Ubuntu%EF%BC%8C%E6%90%AD%E5%BB%BAhadoop%E9%9B%86%E7%BE%A4/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//hm.baidu.com"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="manifest" href="/img/pwa/manifest.json"/><link rel="apple-touch-icon" sizes="180x180" href="/img/pwa/apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="/img/pwa/32.png"/><link rel="icon" type="image/png" sizes="16x16" href="/img/pwa/16.png"/><link rel="mask-icon" href="https://cdn.jsdelivr.net/gh/li1055107552/My-blog@master/img/pwa/safari-pinned-tab.svg" color="#5bbad5"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="/cdn/cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="/cdn/cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.css" media="print" onload="this.media='all'"><script>var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "https://hm.baidu.com/hm.js?8f6becbae63b8ad7610d7ae36666ce2e";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: undefined,
  translate: undefined,
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":false},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '天',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: {"limitCount":50,"languages":{"author":"作者: 细粒丁","link":"链接: ","source":"来源: 细粒丁の小窝","info":"著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。"}},
  lightbox: 'fancybox',
  Snackbar: {"chs_to_cht":"你已切换为繁体","cht_to_chs":"你已切换为简体","day_to_night":"你已切换为深色模式","night_to_day":"你已切换为浅色模式","bgLight":"#49b1f5","bgDark":"#121212","position":"top-center"},
  source: {
    jQuery: '/cdn/cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js',
    justifiedGallery: {
      js: '/cdn/cdn.jsdelivr.net/npm/justifiedGallery/dist/js/jquery.justifiedGallery.min.js',
      css: '/cdn/cdn.jsdelivr.net/npm/justifiedGallery/dist/css/justifiedGallery.min.css'
    },
    fancybox: {
      js: '/cdn/cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.js',
      css: '/cdn/cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: true,
  isanchor: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '虚拟机+Ubuntu，搭建hadoop集群',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2022-03-23 04:26:12'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const fontSizeVal = saveToLocal.get('global-font-size')
    if (fontSizeVal !== undefined) {
      document.documentElement.style.setProperty('--global-font-size', fontSizeVal + 'px')
    }
    
    const detectApple = () => {
      if (GLOBAL_CONFIG_SITE.isHome && /iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><link rel="stylesheet" href="/css/csdn/iconfont.css"><meta name="generator" content="Hexo 5.4.0"></head><body><div id="loading-box"><div class="loading-left-bg"></div><div class="loading-right-bg"></div><div class="spinner-box"><div class="configure-border-1"><div class="configure-core"></div></div><div class="configure-border-2"><div class="configure-core"></div></div><div class="loading-word">加载中...</div></div></div><div id="web_bg"></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src= "" data-lazy-src="/image/avatar.jpg" onerror="onerror=null;src='/image/loading.webp'" alt="avatar"/></div><div class="site-data"><div class="data-item is-center"><div class="data-item-link"><a href="/archives/"><div class="headline">文章</div><div class="length-num">17</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/tags/"><div class="headline">标签</div><div class="length-num">24</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/categories/"><div class="headline">分类</div><div class="length-num">4</div></a></div></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 文章归档</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 文章标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 文章分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('/image/bg9.jpg')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">细粒丁の小窝</a></span><div id="menus"><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 文章归档</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 文章标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 文章分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">虚拟机+Ubuntu，搭建hadoop集群</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2021-03-13T21:29:56.000Z" title="发表于 2021-03-14 05:29:56">2021-03-14</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2022-03-22T20:26:12.339Z" title="更新于 2022-03-23 04:26:12">2022-03-23</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/%E7%B2%BE%E9%80%9A%E5%90%84%E8%BD%AF%E4%BB%B6%E7%9A%84%E5%AE%89%E8%A3%85/">精通各软件的安装</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="虚拟机+Ubuntu，搭建hadoop集群"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="一些说明和资源"><a href="#一些说明和资源" class="headerlink" title="一些说明和资源"></a>一些说明和资源</h1><p>虚拟机：VirtualBox-6.1.18<br>Ubuntu：16.04桌面版<br>JDK：<a target="_blank" rel="noopener" href="https://www.oracle.com/java/technologies/javase/javase-jdk8-downloads.html">jdk-8u281-linux-x64.tar.gz</a><br>hadoop：<a target="_blank" rel="noopener" href="http://archive.apache.org/dist/hadoop/core/hadoop-2.7.3/hadoop-2.7.3.tar.gz">hadoop2.7.3</a><br>可能用到的资源：<a target="_blank" rel="noopener" href="https://download.csdn.net/download/li1055107552/15776196">hadoop-instead</a></p>
<hr>
<h2 id="安装虚拟机和Ubuntu16-04"><a href="#安装虚拟机和Ubuntu16-04" class="headerlink" title="安装虚拟机和Ubuntu16.04"></a>安装虚拟机和Ubuntu16.04</h2><p>传送门：<a target="_blank" rel="noopener" href="https://blog.csdn.net/li1055107552/article/details/114766651">下载及安装教程</a></p>
<blockquote>
<p> 稍有不同的地方：<br><img src= "" data-lazy-src="https://img-blog.csdnimg.cn/20210313233828663.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2xpMTA1NTEwNzU1Mg==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"></p>
</blockquote>
<hr>
<h2 id="安装hadoop"><a href="#安装hadoop" class="headerlink" title="安装hadoop"></a>安装hadoop</h2><br/>

<h4 id="1-配置虚拟机网卡"><a href="#1-配置虚拟机网卡" class="headerlink" title="1. 配置虚拟机网卡"></a>1. 配置虚拟机网卡</h4><pre><code>     1. 打开VirtualBox
     2. 点击右上角 管理 --&gt; 主机网络管理器 --&gt; 创建网卡
    
</code></pre>
<p><img src= "" data-lazy-src="https://img-blog.csdnimg.cn/img_convert/c8c9b55601207655f98a85d995350e45.png#pic_center" alt="在这里插入图片描述"></p>
<br/>

<h4 id="2-配置主机网卡"><a href="#2-配置主机网卡" class="headerlink" title="2. 配置主机网卡"></a>2. 配置主机网卡</h4><p><img src= "" data-lazy-src="https://img-blog.csdnimg.cn/img_convert/f47ddb53bc42e835fb6e7a75c4d37d92.png#pic_center" alt="在这里插入图片描述"></p>
<br/>

<h4 id="3-进入到Ubuntu"><a href="#3-进入到Ubuntu" class="headerlink" title="3.进入到Ubuntu"></a>3.进入到Ubuntu</h4><p><font size=3>以下操作在 hadoop-master 下完成</font></p>
<h5 id="3-1-配置hosts文件"><a href="#3-1-配置hosts文件" class="headerlink" title="3.1 配置hosts文件"></a>3.1 配置hosts文件</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ sudo vim /etc/hosts</span><br></pre></td></tr></table></figure>
<blockquote>
<p>//  配置如下，后面的ipv6部分无需更改：<br/><br><img src= "" data-lazy-src="https://img-blog.csdnimg.cn/20210314030323120.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2xpMTA1NTEwNzU1Mg==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"></p>
</blockquote>
<br/>

<h5 id="3-2-下载和配置Java"><a href="#3-2-下载和配置Java" class="headerlink" title="3.2 下载和配置Java"></a>3.2 下载和配置Java</h5><p>JDK下载：<a target="_blank" rel="noopener" href="https://www.oracle.com/java/technologies/javase/javase-jdk8-downloads.html">官方下载地址</a><br>本地下载：<a target="_blank" rel="noopener" href="https://download.csdn.net/download/li1055107552/15773067">jdk-8u281-linux-x64.tar.gz</a></p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">// 进入到jdk的存放位置 解压</span><br><span class="line">$ tar -xvf jdk-8u281-linux-x64.tar.gz</span><br><span class="line"></span><br><span class="line">// 将文件夹复制到/usr/java 目录下</span><br><span class="line">$ sudo cp -r jdk1.8.0_281/ /usr/java</span><br></pre></td></tr></table></figure>

<p>接着配置环境变量</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ sudo vim /etc/profile</span><br></pre></td></tr></table></figure>
<p>在 profile 中添加以下内容：</p>
<pre><code>export JAVA_HOME=/usr/java
export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH
export JRE_HOME=$JAVA_HOME/jre

# 下面是 后面hadoop的环境变量
# export HADOOP_HOME=/usr/hadoop
# export CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath):$CLASSPATH
# export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
# export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
</code></pre>
<p>然后是保存 并使profile文件立即生效：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ <span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>
<p>接着可以输入以下命令查看是否安装成功：</p>
<pre><code>$ java -version
</code></pre>
<p>若能成功打印版本信息则为成功。</p>
<br/>

<h5 id="3-3-下载和配置Hadoop"><a href="#3-3-下载和配置Hadoop" class="headerlink" title="3.3 下载和配置Hadoop"></a>3.3 下载和配置Hadoop</h5><p>官方地址：<a target="_blank" rel="noopener" href="http://archive.apache.org/dist/hadoop/core/">传送门</a><br>一步到位：<a target="_blank" rel="noopener" href="http://archive.apache.org/dist/hadoop/core/hadoop-2.7.3/hadoop-2.7.3.tar.gz">hadoop-2.7.3.tar.gz</a></p>
<pre><code>整体的布局：
NameNode：hadoop-master
DataNode：hadoop-master、hadoop-slave1、hadoop-slave2
ResourceManager：hadoop-master
NodeManager：hadoop-master
</code></pre>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">// 进入到目录下解压</span><br><span class="line">$ tar -xvf hadoop-2.7.3.tar.gz</span><br><span class="line"></span><br><span class="line">// 将文件夹复制到/usr/hadoop 目录下</span><br><span class="line">$ sudo cp -r hadoop-2.7.3/ /usr/hadoop </span><br></pre></td></tr></table></figure>

<p>配置环境变量</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">// 把上面刚刚Java的环境变量 hadoop部分注释去掉即可</span><br><span class="line">$ sudo vim /etc/profile</span><br><span class="line"></span><br><span class="line">// 同样使profile文件立即生效：</span><br><span class="line">$ <span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>

<p>接下来需要更改几个文件的内容：</p>
<p><img src= "" data-lazy-src="https://img-blog.csdnimg.cn/20210314021000410.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2xpMTA1NTEwNzU1Mg==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>注：mapred-site.xml 没有此文件，将mapred-site.xml.template复制一份改即可</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br></pre></td><td class="code"><pre><span class="line">// hadoop-env.sh 	将java的目录改为绝对路径，以免启动hadoop时找不到Java目录而报错</span><br><span class="line"># The java implementation to use.</span><br><span class="line">export JAVA_HOME=/usr/java</span><br><span class="line"></span><br><span class="line">// slaves 	修改为以下内容，三台主机的名称：</span><br><span class="line">hadoop-master</span><br><span class="line">hadoop-slave1</span><br><span class="line">hadoop-slave2</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">// core-site.xml		</span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>hadoop.tmp.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>file:/usr/hadoop/tmp<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>fs.defaultFS<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>hdfs://hadoop-master:9000<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">// hdfs-site.xml</span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.replication<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>2<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.namenode.secondary.http-address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop-master:50090<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.namenode.name.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>file:/usr/hadoop/tmp/dfs/name<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.datanode.data.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>file:/usr/hadoop/tmp/dfs/data<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">// mapred-site.xml</span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.framework.name<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>yarn<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop-master:10020<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.webapp.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop-master:19888<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">// yarn-site.xml</span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.resourcemanager.hostname<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop-master<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.aux-services<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>mapreduce_shuffle<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.log-aggregation-enable<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>true<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.log-dirs<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>$&#123;yarn.log.dir&#125;/userlogs<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br><span class="line"></span><br></pre></td></tr></table></figure>


<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">故意写在后面：</span><br><span class="line">懒人专属：当你懒到一定程度时，可以直接利用 hadoop-instead 里的文件替换掉 /usr/.../etc/hadoop 里的文件就好了</span><br></pre></td></tr></table></figure>


<p>配置好之后，保存并关闭该主机 hadoop-master 。<br>复制另外两台主机，分别名为hadoop-slave1、hadoop-slave2</p>
<p>复制好之后，启动三台主机。</p>
<br/>

<h5 id="3-4-分别更改三台主机的网络配置"><a href="#3-4-分别更改三台主机的网络配置" class="headerlink" title="3.4 分别更改三台主机的网络配置"></a>3.4 分别更改三台主机的网络配置</h5><br/>

<h5 id="3-4-1-hadoop-master"><a href="#3-4-1-hadoop-master" class="headerlink" title="3.4.1 hadoop-master"></a>3.4.1 hadoop-master</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ ifconfig -a</span><br></pre></td></tr></table></figure>
<p>应该能看到一个网卡是没有ip地址的，那就是需要配置的。（如：enp0s8）</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ sudo vim /etc/network/interfaces</span><br></pre></td></tr></table></figure>
<p>在后面添加以下内容：</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">auto enp0s8</span><br><span class="line">iface enp0s8 inet static</span><br><span class="line">address 192.168.56.1</span><br><span class="line">netmask 255.255.255.0</span><br></pre></td></tr></table></figure>
<blockquote>
<p>// /etc/network/interfaces：<br/><br><img src= "" data-lazy-src="https://img-blog.csdnimg.cn/20210314031531869.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2xpMTA1NTEwNzU1Mg==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"></p>
</blockquote>
<p>开启网卡</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ sudo ifup enp0s8</span><br></pre></td></tr></table></figure>
<p>开启后，再次查看状态，应该是有ip地址的，如192.168.56.1</p>
<br/>

<h5 id="3-4-2-hadoop-slave1、hadoop-slave2"><a href="#3-4-2-hadoop-slave1、hadoop-slave2" class="headerlink" title="3.4.2 hadoop-slave1、hadoop-slave2"></a>3.4.2 hadoop-slave1、hadoop-slave2</h5><p>更改一下设备名称，不然host不认…</p>
<p><img src= "" data-lazy-src="https://img-blog.csdnimg.cn/20210314032451455.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2xpMTA1NTEwNzU1Mg==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br><img src= "" data-lazy-src="https://img-blog.csdnimg.cn/20210314032451368.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2xpMTA1NTEwNzU1Mg==,size_16,color_FFFFFF,t_70#pic_center" alt="在这里插入图片描述"><br>其他操作与 3.4.1 同理。<br>配置好之后，三台主机之间，应该是可以成功ping通的。</p>
<br/>

<h5 id="3-5-配置SSH免密登录"><a href="#3-5-配置SSH免密登录" class="headerlink" title="3.5 配置SSH免密登录"></a>3.5 配置SSH免密登录</h5><figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">// 若没有安装ssh的可以先下载更新一下(所有主机都要)</span><br><span class="line">$ sudo apt-get install openssh-server openssh-client</span><br></pre></td></tr></table></figure>

<p>回到我们的 hadoop-master 主机上</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">$ ssh-keygen -t rsa -P <span class="string">&#x27;&#x27;</span> -f ~/.ssh/id_rsa</span><br><span class="line"></span><br><span class="line">// 分发到不同主机上</span><br><span class="line">$ ssh-copy-id username@hadoop-slave1</span><br><span class="line">$ ssh-copy-id username@hadoop-slave2</span><br><span class="line"></span><br><span class="line">// 本机免密登录</span><br><span class="line">$ <span class="built_in">cd</span> ~/.ssh</span><br><span class="line">$ cat id_rsa.pub &gt;&gt; authorized_keys</span><br><span class="line"></span><br></pre></td></tr></table></figure>
<p>注：链接时提示输入密码，是需要链接的主机的密码，不是本机的密码。<br>连接后，若想要退出链接，可以执行 <code>$exit</code></p>
<hr>
<h2 id="启动hadoop"><a href="#启动hadoop" class="headerlink" title="启动hadoop"></a>启动hadoop</h2><p>三台主机都开启后，在 master主机 上执行指令，格式化HDFS文件系统：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ hdfs namenode -format</span><br></pre></td></tr></table></figure>
<p>在master节点启动hadoop集群</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">$ start-all.sh</span><br><span class="line">或</span><br><span class="line">$ start-dfs.sh</span><br><span class="line">$ start-yarn.sh</span><br></pre></td></tr></table></figure>

<p>然后可以通过<code>$ jps</code> 查看信息，各主机的信息 应该与上文中 3.3 的整体的布局相一致</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">还可以通过以下命令查看hadoop集群系统的状态：</span><br><span class="line">$ hdfs dfsadmin -report</span><br><span class="line"></span><br><span class="line">或直接打开网页，查看更详细的信息：</span><br><span class="line">http://192.168.56.1:50070</span><br></pre></td></tr></table></figure>

<hr>
<h2 id="启动程序"><a href="#启动程序" class="headerlink" title="启动程序"></a>启动程序</h2><p>先跑个demo，算个π</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ hadoop jar /usr/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar pi 10 10</span><br></pre></td></tr></table></figure>
<p>第一次执行，时间会较长，可能需要2-5分钟</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">// 试过一次好像启动的时候报错了，说要把这个关掉，先写在这吧：</span><br><span class="line">$ hdfs dfsadmin -safemode leave</span><br></pre></td></tr></table></figure>

<br/>

<p>其他：（我在另一本书上看到的，大家也可以拿来试着跑一下）</p>
<ul>
<li>日志分析：<br>  github：<a target="_blank" rel="noopener" href="https://github.com/bdintro/bdintro.git">https://github.com/bdintro/bdintro.git</a><br> 编译源代码采用mvn package，测试数据为hadoop-user-datanode-dell119.log.zip<br> 在测试前先把对应数据上传到HDFS集群中，把使用mvn package编译好的jar包 <figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">// 上传日志到hadoop</span><br><span class="line">$ hadoop fs -copyFromLocal hadoop-yangyaru-datanode-dell119.log /</span><br><span class="line"></span><br><span class="line">// 启动</span><br><span class="line">$ hadoop jar 打包好的jar包路径/bigdata-0.0.1.jar bigdata.bigdaba.Grep WARN 需分析的日志路径/hadoop-user-datanode-dell119.log 结果输出的路径/output</span><br></pre></td></tr></table></figure>
</li>
<li>交通流量分析：<br>   github：<a target="_blank" rel="noopener" href="https://github.com/bdintro/bdintro.git">https://github.com/bdintro/bdintro.git</a><br>   详细的交通流量数据格式描述如网站所述：<a target="_blank" rel="noopener" href="https://www.kaggle.com/jana36/us-traffic-violations-montgomery-county-poliet">https://www.kaggle.com/jana36/us-traffic-violations-montgomery-county-poliet</a><br>  编译源代码采用mvn package，测试数据为Traffic_Violations.csv.zip<br>  在测试前先把对应数据上传到HDFS集群中，把使用mvn package编译好的jar包  <figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">// 上传交通记录到hadoop</span><br><span class="line">$ hadoop fs -copyFromLocal Traffic_Violations.csv /</span><br><span class="line"></span><br><span class="line">// 启动</span><br><span class="line">$ hadoop jar 打包好的jar包路径/bigdata-0.0.1.jar bigdata.bigdaba.TrafficTotal WARN 需分析的日志路径/Traffic_Violations.csv 结果输出的路径/output</span><br></pre></td></tr></table></figure></li>
</ul>
<br/>

<hr>
<h2 id="完"><a href="#完" class="headerlink" title="完"></a>完</h2></article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="mailto:undefined">细粒丁</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="https://tenyding.cn/%E7%B2%BE%E9%80%9A%E5%90%84%E8%BD%AF%E4%BB%B6%E7%9A%84%E5%AE%89%E8%A3%85/%E8%99%9A%E6%8B%9F%E6%9C%BA+Ubuntu%EF%BC%8C%E6%90%AD%E5%BB%BAhadoop%E9%9B%86%E7%BE%A4/">https://tenyding.cn/精通各软件的安装/虚拟机+Ubuntu，搭建hadoop集群/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://tenyding.cn" target="_blank">细粒丁の小窝</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/%E8%99%9A%E6%8B%9F%E6%9C%BA/">虚拟机</a><a class="post-meta__tags" href="/tags/Linux/">Linux</a><a class="post-meta__tags" href="/tags/Hadoop/">Hadoop</a></div><div class="post_share"><div class="social-share" data-image="/image/bg9.jpg" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="/cdn/cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="/cdn/cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><div class="post-reward"><div class="reward-button button--animated"><i class="fas fa-qrcode"></i> 打赏</div><div class="reward-main"><ul class="reward-all"><li class="reward-item"><a href="/img/wechat.jpg" target="_blank"><img class="post-qr-code-img" src= "" data-lazy-src="/img/wechat.jpg" alt="微信"/></a><div class="post-qr-code-desc">微信</div></li><li class="reward-item"><a href="/img/alipay.jpg" target="_blank"><img class="post-qr-code-img" src= "" data-lazy-src="/img/alipay.jpg" alt="支付宝"/></a><div class="post-qr-code-desc">支付宝</div></li></ul></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/%E7%B2%BE%E9%80%9A%E5%90%84%E8%BD%AF%E4%BB%B6%E7%9A%84%E5%AE%89%E8%A3%85/VirtualBox%E5%AE%89%E8%A3%85Ubuntu16.04/"><img class="prev-cover" src= "" data-lazy-src="/image/bg9.jpg" onerror="onerror=null;src='/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">VirtualBox安装Ubuntu16.04</div></div></a></div><div class="next-post pull-right"><a href="/%E6%8A%80%E6%9C%AF/JS%E9%97%AD%E5%8C%85%E5%BC%82%E6%AD%A5%E9%97%AE%E9%A2%98/"><img class="next-cover" src= "" data-lazy-src="/image/bg4.jpg" onerror="onerror=null;src='/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">JS闭包异步问题</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span> 相关推荐</span></div><div class="relatedPosts-list"><div><a href="/精通各软件的安装/HBase安装与配置/" title="大型数据库（2）--- HBase安装与配置"><img class="cover" src= "" data-lazy-src="/image/bg4.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2021-09-18</div><div class="title">大型数据库（2）--- HBase安装与配置</div></div></a></div><div><a href="/精通各软件的安装/Hive安装与配置/" title="大型数据库（1）--- Hive安装与配置"><img class="cover" src= "" data-lazy-src="/image/bg7.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2021-09-18</div><div class="title">大型数据库（1）--- Hive安装与配置</div></div></a></div><div><a href="/精通各软件的安装/Mahout安装与配置/" title="大型数据库（3）--- Mahout安装与配置"><img class="cover" src= "" data-lazy-src="/image/bg11.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2021-09-18</div><div class="title">大型数据库（3）--- Mahout安装与配置</div></div></a></div><div><a href="/精通各软件的安装/Kafka安装与配置/" title="大型数据库（8）--- Kafka安装与配置"><img class="cover" src= "" data-lazy-src="/image/bg8.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2021-09-20</div><div class="title">大型数据库（8）--- Kafka安装与配置</div></div></a></div><div><a href="/精通各软件的安装/Spark安装与配置/" title="大型数据库（5）--- Spark安装与配置"><img class="cover" src= "" data-lazy-src="/image/bg4.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2021-09-19</div><div class="title">大型数据库（5）--- Spark安装与配置</div></div></a></div><div><a href="/精通各软件的安装/Sqoop安装与配置/" title="大型数据库（4）--- Sqoop安装与配置"><img class="cover" src= "" data-lazy-src="/image/bg3.jpg" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2021-09-19</div><div class="title">大型数据库（4）--- Sqoop安装与配置</div></div></a></div></div></div><hr/><div id="post-comment"><div class="comment-head"><div class="comment-headline"><i class="fas fa-comments fa-fw"></i><span> 评论</span></div></div><div class="comment-wrap"><div><div id="twikoo-wrap"></div></div></div></div></div><div class="aside-content" id="aside-content"><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%B8%80%E4%BA%9B%E8%AF%B4%E6%98%8E%E5%92%8C%E8%B5%84%E6%BA%90"><span class="toc-number">1.</span> <span class="toc-text">一些说明和资源</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%AE%89%E8%A3%85%E8%99%9A%E6%8B%9F%E6%9C%BA%E5%92%8CUbuntu16-04"><span class="toc-number">1.1.</span> <span class="toc-text">安装虚拟机和Ubuntu16.04</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%AE%89%E8%A3%85hadoop"><span class="toc-number">1.2.</span> <span class="toc-text">安装hadoop</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#1-%E9%85%8D%E7%BD%AE%E8%99%9A%E6%8B%9F%E6%9C%BA%E7%BD%91%E5%8D%A1"><span class="toc-number">1.2.0.1.</span> <span class="toc-text">1. 配置虚拟机网卡</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#2-%E9%85%8D%E7%BD%AE%E4%B8%BB%E6%9C%BA%E7%BD%91%E5%8D%A1"><span class="toc-number">1.2.0.2.</span> <span class="toc-text">2. 配置主机网卡</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#3-%E8%BF%9B%E5%85%A5%E5%88%B0Ubuntu"><span class="toc-number">1.2.0.3.</span> <span class="toc-text">3.进入到Ubuntu</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#3-1-%E9%85%8D%E7%BD%AEhosts%E6%96%87%E4%BB%B6"><span class="toc-number">1.2.0.3.1.</span> <span class="toc-text">3.1 配置hosts文件</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#3-2-%E4%B8%8B%E8%BD%BD%E5%92%8C%E9%85%8D%E7%BD%AEJava"><span class="toc-number">1.2.0.3.2.</span> <span class="toc-text">3.2 下载和配置Java</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#3-3-%E4%B8%8B%E8%BD%BD%E5%92%8C%E9%85%8D%E7%BD%AEHadoop"><span class="toc-number">1.2.0.3.3.</span> <span class="toc-text">3.3 下载和配置Hadoop</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#3-4-%E5%88%86%E5%88%AB%E6%9B%B4%E6%94%B9%E4%B8%89%E5%8F%B0%E4%B8%BB%E6%9C%BA%E7%9A%84%E7%BD%91%E7%BB%9C%E9%85%8D%E7%BD%AE"><span class="toc-number">1.2.0.3.4.</span> <span class="toc-text">3.4 分别更改三台主机的网络配置</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#3-4-1-hadoop-master"><span class="toc-number">1.2.0.3.5.</span> <span class="toc-text">3.4.1 hadoop-master</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#3-4-2-hadoop-slave1%E3%80%81hadoop-slave2"><span class="toc-number">1.2.0.3.6.</span> <span class="toc-text">3.4.2 hadoop-slave1、hadoop-slave2</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#3-5-%E9%85%8D%E7%BD%AESSH%E5%85%8D%E5%AF%86%E7%99%BB%E5%BD%95"><span class="toc-number">1.2.0.3.7.</span> <span class="toc-text">3.5 配置SSH免密登录</span></a></li></ol></li></ol></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%90%AF%E5%8A%A8hadoop"><span class="toc-number">1.3.</span> <span class="toc-text">启动hadoop</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%90%AF%E5%8A%A8%E7%A8%8B%E5%BA%8F"><span class="toc-number">1.4.</span> <span class="toc-text">启动程序</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%AE%8C"><span class="toc-number">1.5.</span> <span class="toc-text">完</span></a></li></ol></li></ol></div></div></div></div></main><footer id="footer" style="background-image: url('/image/bg9.jpg')"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2022 By 细粒丁</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div><div class="footer_custom_text"><a target="_blank" rel="noopener" href="https://beian.miit.gov.cn/"><span>粤ICP备20048893号</span></a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="font-plus" type="button" title="放大字体"><i class="fas fa-plus"></i></button><button id="font-minus" type="button" title="缩小字体"><i class="fas fa-minus"></i></button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><a id="to_comment" href="#post-comment" title="直达评论"><i class="fas fa-comments"></i></a><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="/cdn/cdn.jsdelivr.net/npm/instant.page/instantpage.min.js" type="module"></script><script src="/cdn/cdn.jsdelivr.net/npm/vanilla-lazyload/dist/lazyload.iife.min.js"></script><script src="/cdn/cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.js"></script><script>function panguFn () {
  if (typeof pangu === 'object') pangu.autoSpacingPage()
  else {
    getScript('/cdn/cdn.jsdelivr.net/npm/pangu/dist/browser/pangu.min.js')
      .then(() => {
        pangu.autoSpacingPage()
      })
  }
}

function panguInit () {
  if (true){
    GLOBAL_CONFIG_SITE.isPost && panguFn()
  } else {
    panguFn()
  }
}

document.addEventListener('DOMContentLoaded', panguInit)</script><script>var preloader = {
  endLoading: () => {
    document.body.style.overflow = 'auto';
    document.getElementById('loading-box').classList.add("loaded")
  },
  initLoading: () => {
    document.body.style.overflow = '';
    document.getElementById('loading-box').classList.remove("loaded")

  }
}
window.addEventListener('load',preloader.endLoading())</script><div class="js-pjax"><script>if (document.getElementsByClassName('mermaid').length) {
  if (window.mermaidJsLoad) mermaid.init()
  else {
    getScript('/cdn/cdn.jsdelivr.net/npm/mermaid/dist/mermaid.min.js').then(() => {
      window.mermaidJsLoad = true
      mermaid.initialize({
        theme: 'default',
      })
      false && mermaid.init()
    })
  }
}</script><script>(()=>{
  const $countDom = document.getElementById('twikoo-count')
  const init = () => {
    twikoo.init(Object.assign({
      el: '#twikoo-wrap',
      envId: 'tenyding-17a635',
      region: ''
    }, null))
  }

  const getCount = () => {
    twikoo.getCommentsCount({
      envId: 'tenyding-17a635',
      region: '',
      urls: [window.location.pathname],
      includeReply: false
    }).then(function (res) {
      $countDom.innerText = res[0].count
    }).catch(function (err) {
      console.error(err);
    });
  }

  const loadTwikoo = (bool = false) => {
    if (typeof twikoo === 'object') {
      init()
      bool && $countDom && setTimeout(getCount,0)
    } else {
      getScript('/cdn/cdn.jsdelivr.net/npm/twikoo@1.4.18/dist/twikoo.all.min.js').then(()=> {
        init()
        bool && $countDom && setTimeout(getCount,0)
      })
    }
  }

  if ('Twikoo' === 'Twikoo' || !true) {
    if (true) btf.loadComment(document.getElementById('twikoo-wrap'), loadTwikoo)
    else loadTwikoo(true)
  } else {
    window.loadOtherComment = () => {
      loadTwikoo()
    }
  }
})()</script></div><script id="click-heart" src="/cdn/cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/click-heart.min.js" async="async" mobile="false"></script><link rel="stylesheet" href="/cdn/cdn.jsdelivr.net/npm/aplayer/dist/APlayer.min.css" media="print" onload="this.media='all'"><script src="/cdn/cdn.jsdelivr.net/npm/aplayer/dist/APlayer.min.js"></script><script src="/cdn/cdn.jsdelivr.net/gh/metowolf/MetingJS@1.2/dist/Meting.min.js"></script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div><!-- hexo injector body_end start -->
  <script data-pjax src="/cdn/cdn.jsdelivr.net/gh/Zfour/hexo-github-calendar@1.21/hexo_githubcalendar.js"></script>
  <script data-pjax>
        function GithubCalendarConfig(){
            var git_githubapiurl ="https://python-github-calendar-api.vercel.app/api?li1055107552";
            var git_color =['#ebedf0', '#fdcdec', '#fc9bd9', '#fa6ac5', '#f838b2', '#f5089f', '#c4067e', '#92055e', '#540336', '#48022f', '#30021f'];
            var git_user ="li1055107552";
            var parent_div_git = document.getElementById('recent-posts');
            var git_div_html = ' <div class="recent-post-item" style="width:100%;height:auto;padding:10px;"> <div id="github_loading" style="width:10%;height:100%;margin:0 auto;display: block"> <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"  viewBox="0 0 50 50" style="enable-background:new 0 0 50 50" xml:space="preserve"> <path fill="#d0d0d0" d="M25.251,6.461c-10.318,0-18.683,8.365-18.683,18.683h4.068c0-8.071,6.543-14.615,14.615-14.615V6.461z" transform="rotate(275.098 25 25)"> <animateTransform attributeType="xml" attributeName="transform" type="rotate" from="0 25 25" to="360 25 25" dur="0.6s" repeatCount="indefinite"> </animateTransform> </path> </svg> </div> <div id="github_container"></div> </div>';
            if(parent_div_git && location.pathname =='/'){
                console.log('已挂载github calendar')
                // parent_div_git.innerHTML=git_div_html+parent_div_git.innerHTML // 无报错，但不影响使用(支持pjax跳转)
                parent_div_git.insertAdjacentHTML("afterbegin",git_div_html) // 有报错，但不影响使用(支持pjax跳转)
            };
            GithubCalendar(git_githubapiurl,git_color,git_user)
        }
        if(document.getElementById('recent-posts')){
            GithubCalendarConfig()
        }
    </script>
    <style>#github_container{min-height:280px}@media screen and (max-width:650px) {#github_container{background-image:;min-height:0px}}</style>
    <style></style><!-- hexo injector body_end end --></body></html>